From cd892286fb82d971162f252eb2178e0af13c6a8d Mon Sep 17 00:00:00 2001 From: Jonathan Dieter Date: Tue, 13 Mar 2018 21:10:04 +0200 Subject: [PATCH] Finish code to allow downloading of zck files Signed-off-by: Jonathan Dieter --- include/zck.h | 45 ++++--- src/lib/dl/dl.c | 243 ++++++++++++++++++++++++++++++++++- src/lib/dl/range.c | 33 ++++- src/lib/hash/hash.c | 42 +++++- src/lib/index/index_common.c | 18 ++- src/lib/index/index_create.c | 59 ++++----- src/lib/index/index_read.c | 14 +- src/lib/io.c | 51 ++++++-- src/lib/zck_private.h | 16 ++- src/zck_dl.c | 78 ++--------- 10 files changed, 442 insertions(+), 157 deletions(-) diff --git a/include/zck.h b/include/zck.h index 039ed55..28c1e9a 100644 --- a/include/zck.h +++ b/include/zck.h @@ -24,6 +24,22 @@ typedef enum log_type { ZCK_LOG_DEBUG, ZCK_LOG_INFO, ZCK_LOG_WARNING, ZCK_LOG_ERROR } log_type; +typedef struct zckIndex { + char *digest; + int digest_size; + int finished; + uint64_t start; + size_t length; + struct zckIndex *next; +} zckIndex; + +typedef struct zckIndexInfo { + uint64_t count; + size_t length; + uint8_t hash_type; + uint8_t digest_size; + zckIndex *first; +} zckIndexInfo; typedef struct zckRange { uint64_t start; @@ -37,37 +53,27 @@ typedef struct zckRangeInfo { unsigned int segments; unsigned int max_ranges; zckRange *first; + zckIndexInfo index; } zckRangeInfo; -typedef struct zckIndex { - char *digest; - uint64_t start; - size_t length; - struct zckIndex *next; -} zckIndex; - -typedef struct zckHashType zckHashType; - -typedef struct zckIndexInfo { - uint64_t count; - size_t length; - zckHashType *hash_type; - zckIndex *first; -} zckIndexInfo; - typedef struct zckDLPriv zckDLPriv; +typedef struct zckCtx zckCtx; +typedef struct zckHash zckHash; typedef struct zckDL { size_t dl; size_t ul; + size_t write_in_chunk; + size_t dl_chunk_data; int dst_fd; char *boundary; - zckIndexInfo index; zckRangeInfo info; zckDLPriv *priv; + struct zckCtx *zck; + zckIndex *tgt_check; + zckHash *chunk_hash; } zckDL; -typedef struct zckCtx zckCtx; zckCtx *zck_create(); void zck_free(zckCtx *zck); @@ -86,6 +92,7 @@ zckIndexInfo *zck_get_index(zckCtx *zck); int zck_decompress_to_file (zckCtx *zck, int src_fd, int dst_fd); int zck_set_full_hash_type(zckCtx *zck, uint8_t hash_type); int zck_set_chunk_hash_type(zckCtx *zck, uint8_t hash_type); +int64_t zck_get_predata_length(zckCtx *zck); char *zck_get_index_digest(zckCtx *zck); char *zck_get_full_digest(zckCtx *zck); int zck_get_full_digest_size(zckCtx *zck); @@ -97,6 +104,7 @@ const char *zck_hash_name_from_type(uint8_t hash_type); const char *zck_comp_name_from_type(uint8_t comp_type); int zck_range_calc_segments(zckRangeInfo *info, unsigned int max_ranges); int zck_range_get_need_dl(zckRangeInfo *info, zckCtx *zck_src, zckCtx *zck_tgt); +int zck_dl_copy_src_chunks(zckRangeInfo *info, zckCtx *src, zckCtx *tgt); int zck_range_get_array(zckRangeInfo *info, char **ra); void zck_range_close(zckRangeInfo *info); void zck_set_log_level(log_type ll); @@ -109,5 +117,6 @@ size_t zck_dl_get_bytes_downloaded(zckDL *dl); size_t zck_dl_get_bytes_uploaded(zckDL *dl); int zck_dl_range(zckDL *dl, char *url); char *zck_dl_get_range(unsigned int start, unsigned int end); +int zck_hash_check_full_file(zckCtx *zck, int dst_fd); #endif diff --git a/src/lib/dl/dl.c b/src/lib/dl/dl.c index ed895e7..fd9d7a2 100644 --- a/src/lib/dl/dl.c +++ b/src/lib/dl/dl.c @@ -40,14 +40,120 @@ return False; \ } +int zck_dl_write_zero(zckCtx *tgt, zckIndex *tgt_idx) { + char buf[BUF_SIZE] = {0}; + size_t tgt_data_offset = tgt->preindex_size + tgt->comp_index_size; + size_t to_read = tgt_idx->length; + if(!zck_seek(tgt->fd, tgt_data_offset + tgt_idx->start, SEEK_SET)) + return False; + while(to_read > 0) { + int rb = BUF_SIZE; + if(rb > to_read) + rb = to_read; + if(!zck_write(tgt->fd, buf, rb)) + return False; + to_read -= rb; + } + return True; +} + +int zck_dl_write(zckDL *dl, const char *at, size_t length) { + if(dl->write_in_chunk < length) + length = dl->write_in_chunk; + if(!zck_write(dl->dst_fd, at, length)) + return -1; + dl->write_in_chunk -= length; + return length; +} + +int zck_dl_md_write(zckDL *dl, const char *at, size_t length) { + int wb = 0; + if(dl->write_in_chunk > 0) { + wb = zck_dl_write(dl, at, length); + if(!zck_hash_update(dl->chunk_hash, at, wb)) + return 0; + if(wb < 0) + return 0; + zck_log(ZCK_LOG_DEBUG, "Writing %lu bytes\n", wb); + dl->dl_chunk_data += wb; + } + return wb; +} + +int zck_dl_write_chunk(zckDL *dl) { + if(dl->chunk_hash == NULL) { + zck_log(ZCK_LOG_ERROR, "Chunk hash not initialized\n"); + return False; + } + char *digest = zck_hash_finalize(dl->chunk_hash); + free(dl->chunk_hash); + if(memcmp(digest, dl->tgt_check->digest, dl->tgt_check->digest_size) != 0) { + zck_log(ZCK_LOG_WARNING, + "Downloaded chunk failed hash check\n"); + if(!zck_dl_write_zero(dl->zck, dl->tgt_check)) + return False; + } else { + dl->tgt_check->finished = True; + } + dl->tgt_check = NULL; + dl->chunk_hash = NULL; + free(digest); + return True; +} + int zck_dl_multidata_cb(zckDL *dl, const char *at, size_t length) { - if(dl == NULL) { - zck_log(ZCK_LOG_ERROR, "zckDL not initialized"); + if(dl == NULL || dl->info.index.first == NULL) { + zck_log(ZCK_LOG_ERROR, "zckDL index not initialized\n"); return 0; } - zck_log(ZCK_LOG_DEBUG, "Writing %lu bytes\n", length); - size_t wb = write(dl->dst_fd, at, length); - return wb; + if(dl->zck == NULL || dl->zck->index.first == NULL) { + zck_log(ZCK_LOG_ERROR, "zckCtx index not initialized\n"); + return 0; + } + int wb = zck_dl_md_write(dl, at, length); + if(dl->write_in_chunk == 0) { + /* Check whether we just finished downloading a chunk and verify it */ + if(dl->tgt_check && !zck_dl_write_chunk(dl)) + return False; + zckIndex *idx = dl->info.index.first; + while(idx) { + if(dl->dl_chunk_data == idx->start) { + zckIndex *tgt_idx = dl->zck->index.first; + while(tgt_idx) { + if(tgt_idx->finished) + tgt_idx = tgt_idx->next; + if(idx->length == tgt_idx->length && + memcmp(idx->digest, tgt_idx->digest, + idx->digest_size) == 0) { + dl->tgt_check = tgt_idx; + dl->chunk_hash = zmalloc(sizeof(zckHash)); + if(!zck_hash_init(dl->chunk_hash, + &(dl->zck->chunk_hash_type))) + return 0; + dl->write_in_chunk = idx->length; + size_t offset = dl->zck->preindex_size + + dl->zck->comp_index_size; + if(!zck_seek(dl->dst_fd, offset + tgt_idx->start, + SEEK_SET)) + return 0; + idx = NULL; + tgt_idx = NULL; + } else { + tgt_idx = tgt_idx->next; + } + } + } + if(idx) + idx = idx->next; + } + } + int wb2 = 0; + if(dl->write_in_chunk > 0 && wb < length) { + wb2 = zck_dl_multidata_cb(dl, at+wb, length-wb); + if(wb2 == 0) + return 0; + } + return wb + wb2; } zckDL *zck_dl_init() { @@ -344,6 +450,81 @@ static size_t write_data(void *ptr, size_t l, size_t c, void *dl_v) { return wb; } +int zck_dl_write_and_verify(zckRangeInfo *info, zckCtx *src, zckCtx *tgt, + zckIndex *src_idx, zckIndex *tgt_idx) { + static char buf[BUF_SIZE] = {0}; + + size_t src_data_offset = src->preindex_size + src->comp_index_size; + size_t tgt_data_offset = tgt->preindex_size + tgt->comp_index_size; + size_t to_read = src_idx->length; + if(!zck_seek(src->fd, src_data_offset + src_idx->start, SEEK_SET)) + return False; + if(!zck_seek(tgt->fd, tgt_data_offset + tgt_idx->start, SEEK_SET)) + return False; + zckHash check_hash = {0}; + if(!zck_hash_init(&check_hash, &(src->chunk_hash_type))) + return False; + while(to_read > 0) { + int rb = BUF_SIZE; + if(rb > to_read) + rb = to_read; + if(!zck_read(src->fd, buf, rb)) + return False; + if(!zck_hash_update(&check_hash, buf, rb)) + return False; + if(!zck_write(tgt->fd, buf, rb)) + return False; + to_read -= rb; + } + char *digest = zck_hash_finalize(&check_hash); + /* If chunk is invalid, overwrite with zeros and add to download range */ + if(memcmp(digest, src_idx->digest, src_idx->digest_size) != 0) { + zck_log(ZCK_LOG_WARNING, "Source hash: %s\n", + zck_hash_get_printable(src_idx->digest, &(src->chunk_hash_type))); + zck_log(ZCK_LOG_WARNING, "Target hash: %s\n", + zck_hash_get_printable(digest, &(src->chunk_hash_type))); + if(!zck_dl_write_zero(tgt, tgt_idx)) + return False; + if(!zck_range_add(info, tgt_idx, tgt)) + return False; + } else { + tgt_idx->finished = True; + zck_log(ZCK_LOG_DEBUG, "Writing %lu bytes at %lu\n", tgt_idx->length, + tgt_idx->start); + } + free(digest); + return True; +} + +int zck_dl_copy_src_chunks(zckRangeInfo *info, zckCtx *src, zckCtx *tgt) { + zckIndexInfo *tgt_info = zck_get_index(tgt); + zckIndexInfo *src_info = zck_get_index(src); + zckIndex *tgt_idx = tgt_info->first; + zckIndex *src_idx = src_info->first; + while(tgt_idx) { + int found = False; + src_idx = src_info->first; + + while(src_idx) { + if(tgt_idx->length == src_idx->length && + memcmp(tgt_idx->digest, src_idx->digest, + zck_get_chunk_digest_size(tgt)) == 0) { + found = True; + break; + } + src_idx = src_idx->next; + } + /* Write out found chunk, then verify that it's valid */ + if(found && !zck_dl_write_and_verify(info, src, tgt, src_idx, tgt_idx)) + return False; + if(!found && !zck_range_add(info, tgt_idx, tgt)) + return False; + + tgt_idx = tgt_idx->next; + } + return True; +} + int zck_dl_range(zckDL *dl, char *url) { if(dl == NULL || dl->priv == NULL || dl->info.first == NULL) { zck_log(ZCK_LOG_ERROR, "Struct not defined\n"); @@ -401,6 +582,8 @@ int zck_dl_bytes(zckDL *dl, char *url, size_t bytes, size_t start, return False; } if(start + bytes > *buffer_len) { + zckIndex idx = {0}; + zck_log(ZCK_LOG_DEBUG, "Seeking to end of temporary file\n"); if(lseek(dl->dst_fd, 0, SEEK_END) == -1) { zck_log(ZCK_LOG_ERROR, "Seek to end of temporary file failed: %s\n", @@ -408,8 +591,10 @@ int zck_dl_bytes(zckDL *dl, char *url, size_t bytes, size_t start, return False; } zck_log(ZCK_LOG_DEBUG, "Downloading %lu bytes at position %lu\n", start+bytes-*buffer_len, *buffer_len); + idx.start = *buffer_len; + idx.length = start+bytes-*buffer_len; zck_range_close(&(dl->info)); - zck_range_add(&(dl->info), *buffer_len, start+bytes-1); + zck_range_add(&(dl->info), &idx, NULL); if(!zck_dl_range(dl, url)) return False; zck_range_close(&(dl->info)); @@ -425,10 +610,49 @@ int zck_dl_bytes(zckDL *dl, char *url, size_t bytes, size_t start, return True; } +int zck_zero_bytes(zckDL *dl, size_t bytes, size_t start, size_t *buffer_len) { + char buf[BUF_SIZE] = {0}; + if(start + bytes > *buffer_len) { + zck_log(ZCK_LOG_DEBUG, "Seeking to end of temporary file\n"); + if(lseek(dl->dst_fd, 0, SEEK_END) == -1) { + zck_log(ZCK_LOG_ERROR, "Seek to end of temporary file failed: %s\n", + strerror(errno)); + return False; + } + size_t write = *buffer_len; + while(write < start + bytes) { + size_t wb = BUF_SIZE; + if(write + wb > start + bytes) + wb = (start + bytes) - write; + if(!zck_write(dl->dst_fd, buf, wb)) + return False; + write += wb; + } + zck_log(ZCK_LOG_DEBUG, "Wrote %lu zeros at position %lu\n", start+bytes-*buffer_len, *buffer_len); + *buffer_len = start+bytes; + zck_log(ZCK_LOG_DEBUG, "Seeking to position %lu\n", start); + if(lseek(dl->dst_fd, start, SEEK_SET) == -1) { + zck_log(ZCK_LOG_ERROR, + "Seek to byte %lu of temporary file failed: %s\n", start, + strerror(errno)); + return False; + } + } + return True; +} + int zck_dl_get_header(zckCtx *zck, zckDL *dl, char *url) { size_t buffer_len = 0; size_t start = 0; - + if(zck == NULL) { + zck_log(ZCK_LOG_ERROR, "zckCtx not initialized\n"); + return False; + } + if(dl == NULL) { + zck_log(ZCK_LOG_ERROR, "zckDL not initialized\n"); + return False; + } + zck->fd = dl->dst_fd; if(!zck_dl_bytes(dl, url, 100, start, &buffer_len)) return False; if(!zck_read_initial(zck, dl->dst_fd)) @@ -458,6 +682,11 @@ int zck_dl_get_header(zckCtx *zck, zckDL *dl, char *url) { return False; if(!zck_read_index(zck, dl->dst_fd)) return False; + zckIndexInfo *info = &(dl->info.index); + info->hash_type = zck->index.hash_type; + zck_log(ZCK_LOG_DEBUG, "Writing zeros to rest of file: %llu\n", zck->index.length + zck->comp_index_size + start); + if(!zck_zero_bytes(dl, zck->index.length, zck->comp_index_size + start, &buffer_len)) + return False; return True; } diff --git a/src/lib/dl/range.c b/src/lib/dl/range.c index dde0732..9216686 100644 --- a/src/lib/dl/range.c +++ b/src/lib/dl/range.c @@ -49,10 +49,13 @@ void zck_range_close(zckRangeInfo *info) { next = next->next; free(tmp); } + zck_index_clean(&(info->index)); memset(info, 0, sizeof(zckRangeInfo)); } -zckRange *zck_range_insert_new(zckRange *prev, zckRange *next, uint64_t start, uint64_t end) { +zckRange *zck_range_insert_new(zckRange *prev, zckRange *next, uint64_t start, + uint64_t end, zckRangeInfo *info, + zckIndex *idx, int add_index) { zckRange *new = zmalloc(sizeof(zckRange)); if(!new) { zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", @@ -69,6 +72,12 @@ zckRange *zck_range_insert_new(zckRange *prev, zckRange *next, uint64_t start, u new->next = next; next->prev = new; } + if(add_index) + if(!zck_index_new_chunk(&(info->index), idx->digest, idx->digest_size, + end-start+1, False)) { + free(new); + return NULL; + } return new; } @@ -89,11 +98,20 @@ void zck_range_merge_combined(zckRangeInfo *info) { } } -int zck_range_add(zckRangeInfo *info, uint64_t start, uint64_t end) { - if(!info) { - zck_log(ZCK_LOG_ERROR, "zckRangeInfo not allocated\n"); +int zck_range_add(zckRangeInfo *info, zckIndex *idx, zckCtx *zck) { + if(info == NULL || idx == NULL) { + zck_log(ZCK_LOG_ERROR, "zckRangeInfo or zckIndex not allocated\n"); return False; } + size_t predata_len = 0; + int add_index = False; + if(zck) { + predata_len = zck_get_predata_length(zck); + add_index = True; + } + + size_t start = idx->start + predata_len; + size_t end = idx->start + predata_len + idx->length - 1; zckRange *prev = info->first; for(zckRange *ptr=info->first; ptr;) { prev = ptr; @@ -101,7 +119,8 @@ int zck_range_add(zckRangeInfo *info, uint64_t start, uint64_t end) { ptr = ptr->next; continue; } else if(start < ptr->start) { - if(zck_range_insert_new(ptr->prev, ptr, start, end) == NULL) + + if(zck_range_insert_new(ptr->prev, ptr, start, end, info, idx, add_index) == NULL) return False; if(info->first == ptr) { info->first = ptr->prev; @@ -118,7 +137,7 @@ int zck_range_add(zckRangeInfo *info, uint64_t start, uint64_t end) { } } /* We've only reached here if we should be last item */ - zckRange *new = zck_range_insert_new(prev, NULL, start, end); + zckRange *new = zck_range_insert_new(prev, NULL, start, end, info, idx, add_index); if(new == NULL) return False; if(info->first == NULL) @@ -153,7 +172,7 @@ int zck_range_get_need_dl(zckRangeInfo *info, zckCtx *zck_src, zckCtx *zck_tgt) src_idx = src_idx->next; } if(!found) - if(!zck_range_add(info, tgt_idx->start, tgt_idx->start+tgt_idx->length-1)) + if(!zck_range_add(info, tgt_idx, zck_tgt)) return False; tgt_idx = tgt_idx->next; diff --git a/src/lib/hash/hash.c b/src/lib/hash/hash.c index 3555242..f1bd430 100644 --- a/src/lib/hash/hash.c +++ b/src/lib/hash/hash.c @@ -35,6 +35,7 @@ #include "sha2/sha2.h" static char unknown[] = "Unknown(\0\0\0\0\0"; +static char hash_text[BUF_SIZE] = {0}; const static char *HASH_NAME[] = { "SHA-1", @@ -116,6 +117,32 @@ void zck_hash_close(zckHash *hash) { return; } +/* Returns 1 if full file hash matches, 0 if it doesn't and -1 if failure */ +int zck_hash_check_full_file(zckCtx *zck, int dst_fd) { + if(!zck_seek(dst_fd, zck->preindex_size + zck->comp_index_size, SEEK_SET)) + return -1; + if(!zck_hash_init(&(zck->check_full_hash), &(zck->hash_type))) + return -1; + char buf[BUF_SIZE] = {0}; + zckIndex *idx = zck->index.first; + zck_log(ZCK_LOG_INFO, "Checking full hash\n"); + while(idx) { + size_t to_read = idx->length; + while(to_read > 0) { + size_t rb = BUF_SIZE; + if(rb > to_read) + rb = to_read; + if(!zck_read(dst_fd, buf, rb)) + return -1; + zck_hash_update(&(zck->check_full_hash), buf, rb); + to_read -= rb; + } + idx = idx->next; + } + return zck_validate_file(zck); + return True; +} + char *zck_hash_finalize(zckHash *hash) { if(hash && hash->ctx && hash->type) { if(hash->type->type == ZCK_HASH_SHA1) { @@ -144,4 +171,17 @@ const char *zck_hash_name_from_type(uint8_t hash_type) { return HASH_NAME[hash_type]; } - +const char *zck_hash_get_printable(const char *digest, zckHashType *type) { + if(digest == NULL || type == NULL) { + zck_log(ZCK_LOG_ERROR, + "digest or zckHashType haven't been initialized\n"); + return False; + } + for(int i=0; idigest_size; i++) { + if(snprintf(hash_text + (i*2), 3, "%02x", (unsigned char)digest[i]) < 0) { + zck_log(ZCK_LOG_ERROR, "Unable to generate printable hash\n"); + return NULL; + } + } + return hash_text; +} diff --git a/src/lib/index/index_common.c b/src/lib/index/index_common.c index 19f6fa4..458b2ae 100644 --- a/src/lib/index/index_common.c +++ b/src/lib/index/index_common.c @@ -31,10 +31,13 @@ #include "zck_private.h" -int zck_index_free(zckCtx *zck) { - if(zck->index.first) { +void zck_index_clean(zckIndexInfo *index) { + if(index == NULL) + return; + + if(index->first) { zckIndex *next; - zckIndex *tmp=zck->index.first; + zckIndex *tmp=index->first; while(tmp != NULL) { next = tmp->next; if(tmp->digest) @@ -43,9 +46,11 @@ int zck_index_free(zckCtx *zck) { tmp = next; } } - if(zck->index.hash_type) - free(zck->index.hash_type); - memset(&(zck->index), 0, sizeof(zckIndexInfo)); + memset(index, 0, sizeof(zckIndexInfo)); +} + +void zck_index_free(zckCtx *zck) { + zck_index_clean(&(zck->index)); if(zck->full_hash_digest) { free(zck->full_hash_digest); zck->full_hash_digest = NULL; @@ -62,5 +67,4 @@ int zck_index_free(zckCtx *zck) { free(zck->index_digest); zck->index_digest = NULL; } - return True; } diff --git a/src/lib/index/index_create.c b/src/lib/index/index_create.c index f691091..4463d00 100644 --- a/src/lib/index/index_create.c +++ b/src/lib/index/index_create.c @@ -51,7 +51,7 @@ int zck_index_finalize(zckCtx *zck) { if(zck->index.first) { zckIndex *tmp = zck->index.first; while(tmp) { - index_size += zck->index.hash_type->digest_size + sizeof(uint64_t); + index_size += zck->index.digest_size + sizeof(uint64_t); tmp = tmp->next; } } @@ -59,7 +59,7 @@ int zck_index_finalize(zckCtx *zck) { /* Write index */ index = zmalloc(index_size); index_loc = index; - memcpy(index_loc, &(zck->index.hash_type->type), 1); + memcpy(index_loc, &(zck->index.hash_type), 1); index_loc += 1; index_count = htole64(zck->index.count); memcpy(index_loc, &index_count, sizeof(uint64_t)); @@ -70,8 +70,8 @@ int zck_index_finalize(zckCtx *zck) { zckIndex *tmp = zck->index.first; while(tmp) { uint64_t end = htole64(tmp->start + tmp->length); - memcpy(index_loc, tmp->digest, zck->index.hash_type->digest_size); - index_loc += zck->index.hash_type->digest_size; + memcpy(index_loc, tmp->digest, zck->index.digest_size); + index_loc += zck->index.hash_type; memcpy(index_loc, &end, sizeof(uint64_t)); index_loc += sizeof(uint64_t); tmp = tmp->next; @@ -114,9 +114,14 @@ int zck_index_finalize(zckCtx *zck) { return True; } -int zck_index_new_chunk(zckIndexInfo *index, char *digest, size_t length) { +int zck_index_new_chunk(zckIndexInfo *index, char *digest, int digest_size, + size_t length, int finished) { if(index == NULL) { - zck_log(ZCK_LOG_ERROR, "Invalid index"); + zck_log(ZCK_LOG_ERROR, "Invalid index\n"); + return False; + } + if(digest_size == 0) { + zck_log(ZCK_LOG_ERROR, "Digest size 0 too small\n"); return False; } zckIndex *idx = zmalloc(sizeof(zckIndex)); @@ -125,18 +130,17 @@ int zck_index_new_chunk(zckIndexInfo *index, char *digest, size_t length) { sizeof(zckIndex)); return False; } - if(digest == NULL || length == 0) { - idx->digest = zmalloc(index->hash_type->digest_size); - if(idx->digest == NULL) { - zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", - index->hash_type->digest_size); - return False; - } - } else { - idx->digest = digest; + idx->digest = zmalloc(digest_size); + if(idx->digest == NULL) { + zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", digest_size); + return False; } + if(digest) + memcpy(idx->digest, digest, digest_size); + idx->digest_size = digest_size; idx->start = index->length; idx->length = length; + idx->finished = finished; if(index->first == NULL) { index->first = idx; } else { @@ -150,21 +154,6 @@ int zck_index_new_chunk(zckIndexInfo *index, char *digest, size_t length) { return True; } -int zck_index_add_dl_chunk(zckDL *dl, char *digest, size_t size) { - if(dl == NULL) { - zck_log(ZCK_LOG_ERROR, "Invalid dl context"); - return False; - } - zckIndex *new_index = zmalloc(sizeof(zckIndex)); - if(new_index == NULL) { - zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", - sizeof(zckIndex)); - return False; - } - dl->index.count = dl->index.count + 1; - return True; -} - int zck_index_add_chunk(zckCtx *zck, char *data, size_t size) { zckHash hash; @@ -174,12 +163,13 @@ int zck_index_add_chunk(zckCtx *zck, char *data, size_t size) { } if(size == 0) { - if(!zck_index_new_chunk(&(zck->index), NULL, size)) + if(!zck_index_new_chunk(&(zck->index), NULL, zck->index.digest_size, + size, True)) return False; } else { if(!zck_hash_update(&(zck->full_hash), data, size)) return False; - if(!zck_hash_init(&hash, zck->index.hash_type)) + if(!zck_hash_init(&hash, &(zck->chunk_hash_type))) return False; if(!zck_hash_update(&hash, data, size)) return False; @@ -188,10 +178,11 @@ int zck_index_add_chunk(zckCtx *zck, char *data, size_t size) { if(digest == NULL) { zck_log(ZCK_LOG_ERROR, "Unable to calculate %s checksum for new chunk\n", - zck_hash_name_from_type(zck->index.hash_type->type)); + zck_hash_name_from_type(zck->index.hash_type)); return False; } - if(!zck_index_new_chunk(&(zck->index), digest, size)) + if(!zck_index_new_chunk(&(zck->index), digest, zck->index.digest_size, + size, True)) return False; } return True; diff --git a/src/lib/index/index_read.c b/src/lib/index/index_read.c index bb15786..f4115ac 100644 --- a/src/lib/index/index_read.c +++ b/src/lib/index/index_read.c @@ -37,7 +37,6 @@ int zck_index_read(zckCtx *zck, char *data, size_t size) { char *digest; uint64_t index_size; uint64_t index_count; - uint8_t hash_type; char *dst = NULL; size_t dst_size = 0; char *cur_loc; @@ -90,6 +89,7 @@ int zck_index_read(zckCtx *zck, char *data, size_t size) { zck->hash_type.digest_size); return False; } + uint8_t hash_type; memcpy(&hash_type, dst, 1); if(!zck_set_chunk_hash_type(zck, hash_type)) { if(dst) @@ -98,7 +98,7 @@ int zck_index_read(zckCtx *zck, char *data, size_t size) { } if((dst_size - (zck->hash_type.digest_size + sizeof(uint64_t)+ 1)) % - (zck->index.hash_type->digest_size + sizeof(uint64_t)) != 0) { + (zck->index.digest_size + sizeof(uint64_t)) != 0) { zck_log(ZCK_LOG_ERROR, "Index size is invalid\n"); if(dst) free(dst); @@ -120,17 +120,19 @@ int zck_index_read(zckCtx *zck, char *data, size_t size) { } uint64_t end = 0; - new->digest = zmalloc(zck->index.hash_type->digest_size); + new->digest = zmalloc(zck->index.digest_size); if(!new->digest) { zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", - zck->index.hash_type->digest_size); + zck->index.digest_size); return False; } - memcpy(new->digest, cur_loc, zck->index.hash_type->digest_size); - cur_loc += zck->index.hash_type->digest_size; + memcpy(new->digest, cur_loc, zck->index.digest_size); + new->digest_size = zck->index.digest_size; + cur_loc += zck->index.digest_size; memcpy(&end, cur_loc, sizeof(uint64_t)); new->start = prev_loc; new->length = le64toh(end) - prev_loc; + new->finished = False; prev_loc = le64toh(end); zck->index.length += new->length; cur_loc += sizeof(uint64_t); diff --git a/src/lib/io.c b/src/lib/io.c index cd0313d..880be94 100644 --- a/src/lib/io.c +++ b/src/lib/io.c @@ -26,44 +26,79 @@ #include #include +#include #include +#include #include #include "zck_private.h" -#define BLK_SIZE 32768 - int zck_read(int fd, char *data, size_t length) { if(length == 0) return True; - - if(read(fd, data, length) != length) { + if(data == NULL) { + zck_log(ZCK_LOG_ERROR, "Unable to read to NULL data pointer\n"); + return False; + } + ssize_t read_bytes = read(fd, data, length); + if(read_bytes == -1) { + zck_log(ZCK_LOG_ERROR, "Error reading data: %s\n", strerror(errno)); + return False; + } else if(read_bytes != length) { zck_log(ZCK_LOG_ERROR, "Short read\n"); return False; } - return True; } int zck_write(int fd, const char *data, size_t length) { if(length == 0) return True; + if(data == NULL) { + zck_log(ZCK_LOG_ERROR, "Unable to write from NULL data pointer\n"); + return False; + } + ssize_t write_bytes = write(fd, data, length); + if(write_bytes == -1) { + zck_log(ZCK_LOG_ERROR, "Error write data: %s\n", strerror(errno)); + return False; + } else if(write_bytes != length) { + zck_log(ZCK_LOG_ERROR, "Short write\n"); + return False; + } + return True; +} + +int zck_seek(int fd, off_t offset, int whence) { + if(lseek(fd, offset, whence) == -1) { + char *wh_str = NULL; - if(write(fd, data, length) != length) + if(whence == SEEK_CUR) { + wh_str = "from current position"; + } else if(whence == SEEK_END) { + wh_str = "from end of file"; + } else if(whence == SEEK_SET) { + wh_str = "from beginning of file"; + } else { + wh_str = "using unknown measurement"; + } + zck_log(ZCK_LOG_ERROR, "Unable to seek to %lu %s: %s\n", offset, wh_str, + strerror(errno)); return False; + } return True; } int zck_chunks_from_temp(zckCtx *zck) { int read_count; - char *data = zmalloc(BLK_SIZE); + char *data = zmalloc(BUF_SIZE); if(data == NULL) return False; if(lseek(zck->temp_fd, 0, SEEK_SET) == -1) return False; - while((read_count = read(zck->temp_fd, data, BLK_SIZE)) > 0) { + while((read_count = read(zck->temp_fd, data, BUF_SIZE)) > 0) { if(read_count == -1 || !zck_write(zck->fd, data, read_count)) { free(data); return False; diff --git a/src/lib/zck_private.h b/src/lib/zck_private.h index 28de9ac..a3df140 100644 --- a/src/lib/zck_private.h +++ b/src/lib/zck_private.h @@ -4,6 +4,8 @@ #include #include +#define BUF_SIZE 32768 + #define zmalloc(x) calloc(1, x) struct zckComp; @@ -25,7 +27,7 @@ typedef struct zckHashType { int digest_size; } zckHashType; -typedef struct { +typedef struct zckHash { zckHashType *type; void *ctx; } zckHash; @@ -69,6 +71,7 @@ typedef struct zckCtx { int temp_fd; int fd; + size_t preindex_size; char *full_hash_digest; char *comp_index; size_t comp_index_size; @@ -78,10 +81,12 @@ typedef struct zckCtx { zckHash check_full_hash; zckComp comp; zckHashType hash_type; + zckHashType chunk_hash_type; } zckCtx; const char *zck_hash_name_from_type(uint8_t hash_type); int zck_get_tmp_fd(); +int zck_validate_file(zckCtx *zck); /* comp/comp.c */ int zck_comp_init(zckCtx *zck); @@ -97,15 +102,20 @@ int zck_hash_init(zckHash *hash, zckHashType *hash_type); int zck_hash_update(zckHash *hash, const char *message, const size_t size); char *zck_hash_finalize(zckHash *hash); void zck_hash_close(zckHash *hash); +const char *zck_hash_get_printable(const char *digest, zckHashType *type); /* index/index.c */ int zck_index_read(zckCtx *zck, char *data, size_t size); int zck_index_finalize(zckCtx *zck); +int zck_index_new_chunk(zckIndexInfo *index, char *digest, int digest_size, + size_t length, int finished); int zck_index_add_chunk(zckCtx *zck, char *data, size_t size); -int zck_index_free(zckCtx *zck); +void zck_index_clean(zckIndexInfo *index); +void zck_index_free(zckCtx *zck); int zck_write_index(zckCtx *zck); /* io.c */ +int zck_seek(int fd, off_t offset, int whence); int zck_read(int fd, char *data, size_t length); int zck_write(int fd, const char *data, size_t length); int zck_chunks_from_temp(zckCtx *zck); @@ -121,7 +131,7 @@ int zck_write_header(zckCtx *zck); /* dl/range.c */ char *zck_range_get_char(zckRange **range, int max_ranges); -int zck_range_add(zckRangeInfo *info, uint64_t start, uint64_t end); +int zck_range_add(zckRangeInfo *info, zckIndex *idx, zckCtx *zck); /* log.c */ void zck_log(log_type lt, const char *format, ...); diff --git a/src/zck_dl.c b/src/zck_dl.c index 650a880..614b13c 100644 --- a/src/zck_dl.c +++ b/src/zck_dl.c @@ -61,11 +61,11 @@ int main (int argc, char *argv[]) { printf("Unable to open %s\n", argv[1]); exit(1); } - close(src_fd); zckDL *dl = zck_dl_init(); if(dl == NULL) exit(1); + dl->zck = zck_tgt; dl->dst_fd = zck_get_tmp_fd(); if(dl->dst_fd < 0) @@ -74,7 +74,7 @@ int main (int argc, char *argv[]) { exit(1); zck_range_close(&(dl->info)); - if(!zck_range_get_need_dl(&(dl->info), zck_src, zck_tgt)) + if(!zck_dl_copy_src_chunks(&(dl->info), zck_src, zck_tgt)) exit(1); int max_ranges = 256; if(!zck_range_calc_segments(&(dl->info), max_ranges)) @@ -97,73 +97,19 @@ int main (int argc, char *argv[]) { free(outname_full);*/ printf("Downloaded %lu bytes\n", zck_dl_get_bytes_downloaded(dl)); + switch(zck_hash_check_full_file(dl->zck, dl->dst_fd)) { + case -1: + exit(1); + break; + case 0: + exit(1); + break; + default: + break; + } zck_dl_free(dl); zck_free(zck_tgt); zck_free(zck_src); zck_dl_global_cleanup(); exit(0); - - /* - if(zck_get_chunk_hash_type(zck_tgt) != zck_get_chunk_hash_type(zck_src)) { - printf("ERROR: Chunk hash types don't match:\n"); - printf(" %s: %s\n", argv[1], zck_hash_name_from_type(zck_get_chunk_hash_type(zck_tgt))); - printf(" %s: %s\n", argv[2], zck_hash_name_from_type(zck_get_chunk_hash_type(zck_src))); - return 1; - } - zckIndex *tgt_idx = zck_get_index(zck_tgt); - zckIndex *src_idx = zck_get_index(zck_src); - if(memcmp(tgt_idx->digest, src_idx->digest, zck_get_chunk_digest_size(zck_tgt)) != 0) - printf("WARNING: Dicts don't match\n"); - int dl_size = 0; - int total_size = 0; - int matched_chunks = 0; - while(tgt_idx) { - int found = False; - src_idx = zck_get_index(zck_src); - - while(src_idx) { - if(memcmp(tgt_idx->digest, src_idx->digest, zck_get_chunk_digest_size(zck_tgt)) == 0) { - found = True; - break; - } - src_idx = src_idx->next; - } - if(!found) { - dl_size += tgt_idx->length; - } else { - matched_chunks += 1; - } - total_size += tgt_idx->length; - tgt_idx = tgt_idx->next; - } - printf("Would download %i of %i bytes\n", dl_size, total_size); - printf("Matched %i of %lu chunks\n", matched_chunks, zck_get_index_count(zck_tgt)); - - zckRangeInfo info = {0}; - if(!zck_range_get_need_dl(&info, zck_src, zck_tgt)) - exit(1); - zckRange *tmp = info.first; - while(tmp) { - printf("Range: %lu - %lu\n", tmp->start, tmp->end); - tmp = tmp->next; - } - printf("Count: %u\n", info.count); - zck_range_calc_segments(&info, 5); - char **ra = calloc(sizeof(char*), info.segments); - if(!ra) { - printf("Unable to allocate %lu bytes\n", sizeof(char*) * info.segments); - exit(1); - } - if(!zck_range_get_array(&info, ra)) - exit(1); - for(int i=0; i